#!/bin/bash

set -x
set -e

# 优化，仅优化编译已有的参数、不加 OpenMP、使用 CUDA 编译依赖库
# 用于进行 perf 分析
CompileMode=optparamscuda

ROOT_PATH=`cd $(dirname $0) && pwd`
SCRIPT_NAME=`basename $0`
SCRIPT=$ROOT_PATH/`basename $0`
cd ${ROOT_PATH}

module purge
# module load icc/2023.1.0
# module load openmpi/4.1.6-mlnxofed-cuda12.3-gdrcopy-full
# module load nvhpc/23.11-cuda11.8
# module load nvhpc-nompi/23.11
# module load cuda/12.3
module load nvhpc/23.11
# module load compiler/2023.1.0
# module load mpi/2021.11
# module load mkl/2023.1.0
module load cmake/3.22
export OMPI_CC=nvc
export OMPI_CXX=nvc++
export OMPI_FC=nvfortran

mkdir -p ${ROOT_PATH}/softwares

# Device Openmp \ CUDA \ OpenMP 三选一

# lapack
# cd ${ROOT_PATH}/lapack-3.11
# sed -i 's#CC = .*#CC = gcc#g' make.inc
# sed -i 's#FC = .*#FC = gfortran#g' make.inc
# make blaslib -j 16
# make cblaslib -j 16
# make lapacklib -j 16
# make lapackelib -j 16

# parmetis
# cd ${ROOT_PATH}/parmetis-4.0.3
# make config cc=mpicc openmp=1 prefix=${ROOT_PATH}/softwares/parmetis/4.0.3/${CompileMode}
# make -j 16
# make install

# kokkos
# cd ${ROOT_PATH}/kokkos-4.2.00
# rm -rf build && mkdir -p build && cd build
# cmake .. -DCMAKE_INSTALL_PREFIX="${ROOT_PATH}/softwares/kokkos/4.2.00/${CompileMode}" \
#     -DCMAKE_CXX_COMPILER=nvc++ \
#     -DKokkos_ENABLE_CUDA=on

# make -j 16 && make install

# hypre
# cd ${ROOT_PATH}/hypre-2.30.0/src

# # # rm -rf build && mkdir -p build && cd build

# # ## configure
# # ### shoule link math_libs/12.3/include to math_libs/include && math_libs/12.3/lib64 to math_libs/lib64
# export CUDA_HOME=${NVHPC_ROOT}/cuda

# # # CC=mpicc CXX=mpic++ CUCC="nvcc" ./configure --prefix="${ROOT_PATH}/softwares/hypre/git/${CompileMode}" --with-MPI --enable-shared --with-openmp --with-cuda --enable-cublas --enable-curand --enable-cusparse --with-extra-CFLAGS="-march=native -mtune=native -fast" --with-extra-CXXFLAGS="-march=native -mtune=native -fast" --with-gpu-arch=80 --with-extra-CUFLAGS="-ccbin mpic++"

# # CC=mpicc CXX=mpic++ CUCC="nvcc" --with-extra-CUFLAGS="-ccbin mpic++ -Xcompiler -fPIC" --enable-debug --with-extra-CFLAGS="-fPIC" --with-extra-CXXFLAGS="-fPIC" 

# ./configure --prefix="${ROOT_PATH}/softwares/hypre/2.30.0/${CompileMode}" --with-MPI --with-cuda --enable-unified-memory --enable-shared --enable-debug --with-gpu-arch=80 --with-kokkos="${ROOT_PATH}/softwares/kokkos/4.2.00/${CompileMode}"

# # # CC=mpicc CXX=mpic++ CUCC="nvcc" ./configure --prefix="${ROOT_PATH}/softwares/hypre/2.29.0/${CompileMode}" --with-MPI --enable-shared --with-openmp --with-cuda --with-gpu-arch=80 --with-extra-CUFLAGS="-ccbin mpic++"

# # # cmake .. -DCMAKE_INSTALL_PREFIX="${ROOT_PATH}/softwares/hypre/2.29.0/${CompileMode}" -DHYPRE_WITH_MPI=on -DHYPRE_ENABLE_SHARED=on -DHYPRE_WITH_OPENMP=on -DHYPRE_WITH_CUDA=on -DHYPRE_ENABLE_UNIFIED_MEMORY=on -DHYPRE_CUDA_SM=80 -DHYPRE_BUILD_EXAMPLES=on

# # #CC=mpicc CXX=mpic++ ./configure --prefix="${ROOT_PATH}/softwares/hypre/2.29.0/${CompileMode}" --enable-shared --with-cuda --with-MPI --with-gpu-arch=80

# # # cmake .. -DCMAKE_INSTALL_PREFIX="${ROOT_PATH}/softwares/hypre/git/${CompileMode}" -DHYPRE_WITH_MPI=on -DCMAKE_BUILD_TYPE=Release -DHYPRE_ENABLE_SHARED=on -DHYPRE_WITH_OPENMP=on -DHYPRE_WITH_CUDA=on -DHYPRE_ENABLE_UNIFIED_MEMORY=on -DHYPRE_ENABLE_CUBLAS=on -DHYPRE_ENABLE_CURAND=on -DHYPRE_ENABLE_CUSPARSE=on -DHYPRE_WITH_EXTRA_CFLAGS="-march=native -mtune=native -O3" -DHYPRE_WITH_EXTRA_CXXFLAGS="-march=native -mtune=native -O3" -DHYPRE_CUDA_SM=80 -DHYPRE_ENABLE_DEVICE_MALLOC_ASYNC=on -DHYPRE_BUILD_EXAMPLES=on
# # make use_cuda=1 -j 16
# make -j 16
# # # make ex1 ex11 ex12 ex13 ex14 ex15 ex16 ex17 ex18 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 use_cuda=1 -j 16
# # ## install
# make install

# petsc
cd ${ROOT_PATH}/petsc-3.20.2

export PETSC_DIR=${ROOT_PATH}/petsc-3.20.2
export PETSC_ARCH=x86_64

sed -i 's/NVCC_WRAPPER_DEFAULT_COMPILER.*/NVCC_WRAPPER_DEFAULT_COMPILER:-"nvc++"}/g' ${ROOT_PATH}/softwares/kokkos/4.2.00/${CompileMode}/bin/nvcc_wrapper

./configure --with-cudac=nvc++ \
    --with-cc=mpicc \
    --with-cxx=mpic++ \
    --with-fc=mpifort \
    --prefix="${ROOT_PATH}/softwares/petsc/3.20.2/${CompileMode}" \
    --with-fortran-bindings=0 \
	--with-hypre-dir="${ROOT_PATH}/softwares/hypre/2.30.0/${CompileMode}" \
    --with-debugging=1 \
    --with-mpi=1 \
    --with-cuda=1 \
    --with-openmp=1 \
    --with-kokkos=1 \
    --with-kokkos-dir="${ROOT_PATH}/softwares/kokkos/4.2.00/${CompileMode}"

make -j 20 PETSC_DIR=${PETSC_DIR} PETSC_ARCH=${PETSC_ARCH} all
make all check

# petsc_solver
# cd ${ROOT_PATH}/petsc_solver

# export CC=mpicc
# export CXX=mpic++
# export CPATH=${ROOT_PATH}/lapack-3.11/CBLAS/include:${ROOT_PATH}/lapack-3.11/LAPACKE/include:$CPATH

# sed -i "s#set(PETSC_DIR .*)#set(PETSC_DIR ${ROOT_PATH}/petsc-3.20.2)#g" CMakeLists.txt
# sed -i "s#set(PETSC_ARCH .*)#set(PETSC_ARCH ${PETSC_ARCH})#g" CMakeLists.txt

# rm -rf build
# mkdir build
# cd build
# cmake .. -DCMAKE_BUILD_TYPE=Release
# make
# mv libpetsc_solver.a ../lib/

# # OpenCAEPoro
# cd ${ROOT_PATH}/OpenCAEPoro

# export CC=mpicc
# export CXX=mpic++

# ## users specific directory paths
# export PARMETIS_DIR=${ROOT_PATH}/parmetis-4.0.3
# export PARMETIS_BUILD_DIR=${PARMETIS_DIR}/build/Linux-x86_64
# export METIS_DIR=${ROOT_PATH}/parmetis-4.0.3/metis
# export METIS_BUILD_DIR=${PARMETIS_DIR}/build/Linux-x86_64
# export PETSC_DIR=${ROOT_PATH}/petsc-3.20.2
# export PETSC_ARCH=${PETSC_ARCH}
# export PETSCSOLVER_DIR=${ROOT_PATH}/petsc_solver
# export CPATH=${ROOT_PATH}/petsc-3.20.2/include/:$CPATH
# export CPATH=${ROOT_PATH}/petsc-3.20.2/${PETSC_ARCH}/include/:${ROOT_PATH}/parmetis-4.0.3/metis/include:${ROOT_PATH}/parmetis-4.0.3/include:$CPATH
# export CPATH=${ROOT_PATH}/lapack-3.11/CBLAS/include/:$CPATH

# ## install
# rm -rf build; mkdir build; cd build;

# echo "cmake -DUSE_PETSCSOLVER=ON -DUSE_PARMETIS=ON -DUSE_METIS=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_BUILD_TYPE=Release -DUSE_OPENMP=ON .."
# cmake -DUSE_PETSCSOLVER=ON -DUSE_PARMETIS=ON -DUSE_METIS=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_BUILD_TYPE=Release -DUSE_OPENMP=ON ..

# echo "make -j 32"
# make -j 32

# echo "make install"
# make install

# link OpenCAEPoro
# mkdir -p ${ROOT_PATH}/softwares/OpenCAEPoro/${CompileMode}/lib
# mkdir -p ${ROOT_PATH}/softwares/OpenCAEPoro/${CompileMode}/data
# cp ${ROOT_PATH}/OpenCAEPoro/lib/libOpenCAEPoroX.a ${ROOT_PATH}/softwares/OpenCAEPoro/${CompileMode}/lib/
# cp ${ROOT_PATH}/OpenCAEPoro/testOpenCAEPoro ${ROOT_PATH}/softwares/OpenCAEPoro/${CompileMode}/
# cp -r ${ROOT_PATH}/OpenCAEPoro/data/* ${ROOT_PATH}/softwares/OpenCAEPoro/${CompileMode}/data
